#!/usr/bin/env python

import ezp_cfg
import glob
import os
import pdb
import pwd
import re
import socket
import subprocess
import sys
import time
import traceback as tb
import xmlrpclib

from ConfigParser import InterpolationMissingOptionError, NoOptionError
from optparse import *
from ezp_java import *
from ezp_util import *
from urllib2 import urlparse

REMOTE_HOST = 'esg-vm-demo01.ccs.ornl.gov'
REMOTE_PORT = 40001
LANDING_PAD = '/tmp/esg/lpad'
ESG_DATAROOT = '/esg/data'
THREDDS_ROOT = '/esg/content/thredds/esgcet'

# ---------------------------------------------------------------------------
def main(args):
    """
    Called as                Behavior
     ezpub.py -L              create symlink ezpub -> ezpub.py
     ezpub.py                 generate help output
     ezpub ...                run dispatch()
    """
    sname = sys.argv[0]
    pname = re.sub('.py$', '', sname)
    if sname.endswith('.py') and not os.path.exists(pname) and "-L" in args:
        print('creating symlink: %s -> %s' % (os.path.basename(pname), sname))
        os.symlink(os.path.basename(sname), pname)
    elif sys._getframe(1).f_code.co_name in ['?', '<module>']:
        if sname.endswith('.py'):
            print("Type 'ezpub <subcommand>' to run one of the following commands.")
            ez_help([])
        else:
            dispatch(sys.argv)

# ---------------------------------------------------------------------------
def dispatch(argv):
    """
    Call ez_help() or a subcommand function.

    The command line "ezpub foober" will call subcommand ez_foober.
    """
    if len(argv) < 2:
        ez_help([])
    elif argv[1] == 'help':
        ez_help(argv[2:])
    else:
        fcnname = "ez_%s" % argv[1]
        if fcnname in dir(sys.modules['__main__']):
            eval("sys.modules['__main__'].%s(argv[2:])" % fcnname)
        else:
            print("unrecognized subfunction: %s" % argv[1])
            ez_help([])
        
# ---------------------------------------------------------------------------
def ez_help(argv):
    """help - show this list

    usage: ezpub help [function-name]

    If a function name is given, display help information for the
    function. Otherwise, show a list of functions with a one line
    description of each one.

    ezpub [function-name] -h

    Show the options available for [function-name].
    """
    d = dir(sys.modules['__main__'])
    prefix = 'ez'
    if 0 < len(argv):
        if '%s_%s' % (prefix, argv[0]) in d:
            dname = "sys.modules['__main__'].%s_%s.__doc__" % (prefix, argv[0])
            x = eval(dname)
            print x
            return
        else:
            print("Subcommand '%s' not recognized. Available subcommands are:"
                  % argv[0])
        
    for o in d:
        if o.startswith(prefix + '_'):
            f = o.replace(prefix + '_', '')
            x = eval("sys.modules['__main__'].%s.__doc__" % o)
            docsum = x.split('\n')[0]
            print "   %s" % (docsum)

# ---------------------------------------------------------------------------
def ez_cleanup(argv):
    """cleanup - remove work files created by publish requests

    usage: ezpub cleanup [--dry-run/-n] [--dir/-D <dir>]
           
    Remove catalog.xml and 1/*.xml under <dir>. If no directory is
    specified, the cleanup will be done on $HOME/.scigest, which is
    usually what you want.

    If --dry-run or -n is specified, nothing will be deleted, but the
    files that would have deleted are listed.
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    p.add_option('-D', '--dir',
                 action='store', default='', dest='dir',
                 help='path to files to clean up')
    p.add_option('-n', '--dry-run',
                 action='store_true', default=False, dest='dryrun',
                 help='report what would happen without doing it')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()

    count = verbose_rm("%s/.scigest/catalog.txt" % user_home(), o.dryrun)

    for filename in glob.glob("%s/.scigest/copylist*.txt" % user_home()):
        count += verbose_rm(filename, o.dryrun)

    count += verbose_rm("%s/.scigest/application.properties"
                        % user_home(), o.dryrun)
    
    for (dir, dl, fl) in os.walk("%s/.scigest/output" % user_home(), False):
        for fn in fl:
            count += verbose_rm("%s/%s" % (dir, fn), o.dryrun)
        count += verbose_rmdir(dir, o.dryrun)

    if count == 0:
        print("Nothing to clean up.")

# ---------------------------------------------------------------------------
def ez_deploy(argv):
    """deploy - move files from the remote landing pad to their final place

    usage: ezpub deploy -i <input-dir> -r <request> [-t <thredds>] [-H <host>]

    This routine deploys files that have already been staged to the
    esgf portal node.
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    p.add_option('-i', '--input',
                 action='store', default='', dest='input',
                 help='path to input file(s)')
    p.add_option('-r', '--reqdir',
                 action='store', default='', dest='reqdir',
                 help='request directory to deploy from')
    p.add_option('-t', '--thredds',
                 action='store', default='', dest='thredds',
                 help='where to find the thredds file to copy')
    p.add_option('-H', '--host',
                 action='store', default='', dest='host',
                 help='which esgf portal to talk to')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()

    # complain if $SCIGEST_HOME or $HOME is not set
    scigest_home()
    user_home()
    
    # get input dir from ~/.scigest/application.properties unless user
    # overrides
    if o.input == '':
        idir = ezp_cfg.read_app_props("data.src")
    else:
        idir = o.input
    
    # get thredds list from ~/.scigest/catalog.txt unless user overrides
    if o.thredds == '':
        t = contents("%s/.scigest/catalog.txt" % user_home(), False)
        thredds = ",".join(["%s/.scigest/output/1/%s" % (user_home(), x.strip())
                            for x in t])
    else:
        thredds = o.thredds
    
    # get target host from ~/.scigest/application.properties
    if o.host == '':
        url = ezp_cfg.read_app_props("esg.search.solr.publish.url")
        (s, loc, path, params, query, fragment) = urlparse.urlparse(url)
        if ':' in loc:
            (host, port) = loc.split(':')
        else:
            host = loc
    else:
        host = o.host
        
    validate_input_dir(idir)
    validate_thredds_list(thredds)
    validate_reqdir(o.reqdir)
    validate_hostname(host)
    
    complete_stage(o.reqdir, idir, thredds, host)
    
# ---------------------------------------------------------------------------
def ez_generate(argv):
    """generate - collect metadata from .nc/.cdf files

    usage: ezpub generate -i <input> -p <project> [-H <hostname>]
           
    Prompt for needed information, scan .nc and .cdf files for
    metadata, and format the metadata into THREDDS .xml file (also
    called a catalog).
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    p.add_option('-i', '--input',
                 action='store', default='', dest='input',
                 help='path to input file(s)')
    p.add_option('-p', '--project',
                 action='store', default='', dest='project',
                 help='specify the project to publish under')
    p.add_option('-u', '--ustest',
                 action='store', default='', dest='ustest',
                 help='for testing ')
    p.add_option('-H', '--host',
                 action='store', default=REMOTE_HOST, dest='host',
                 help='which esgf portal to talk to')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()

    # if o.ustest has been specified, we're just testing userSelect()
    if o.ustest != '':
        us = o.ustest.split(',')
        val = userSelect(us[0], us[1:])
        print("val = %s" %val)
        sys.exit(0)

    # Load esg.ini
    esg_cfg = ezp_cfg.loadConfig("%s/ezpub/esg.ini" % scigest_home())
    
    # if we're just listing projects, do that and exit
    projlist = ezp_cfg.get_value_list('project', 'DEFAULT', esg_cfg)
    
    # Load the models for each project
    mtable = ezp_cfg.loadModels(esg_cfg)
    
    # Figure out the input directory
    default_input = "."
    if o.input == '':
        inpath = interruptible_input("Input path? [%s] > " % default_input)
        inpath = os.path.expandvars(os.path.expanduser(inpath))
        if inpath == '':
            inpath = default_input
    else:
        inpath = o.input

    validate_input_dir(inpath)
    if not os.path.exists(inpath):
        raise StandardError("Input '%s' does not exist" % inpath)
    elif not os.access(inpath, os.R_OK):
        raise StandardError("Input '%s' is not readable" % inpath)

    #
    # Get the project either from the command line or ask the user
    # to specify
    #
    if o.project != '':
        project = o.project
        if project not in projlist:
            raise StandardError("Project '%s' is not recognized" % project)
    else:
        project = userSelect('project', projlist)

    # print("project: %s" % project)
    section = 'project:' + project
    esg_cfg.set(section, 'project', project)
    
    #
    # here we write out the properties file to tell scigest things
    # like project name, etc.
    #
    require_dot_scigest()
    write_application_properties(inpath, project, esg_cfg, o.host)
    
    #
    # here we set up the output directory structure and generate
    # an empty catalog.xml file if needed
    #
    if not os.path.exists("%s/.scigest/output/1" % user_home()):
        os.makedirs("%s/.scigest/output/1" % user_home())
    if not os.path.exists("%s/.scigest/output/catalog.xml" % user_home()):
        make_empty_catalog()

    #
    # call scigest.core.CatalogGen
    #
    build_path = "%s/build" % scigest_home()
    jar_path = "%s/lib" % scigest_home()
    resource_path = "%s/resources" % scigest_home()

    if 0 < len(glob.glob("%s/scigest-*.jar" % jar_path)):
        os.environ['CLASSPATH'] = "%s:%s" % (jar_path, resource_path)
        ext_dirs = jar_path
        print("adding %s, %s to class path" % (jar_path, resource_path))
    elif os.path.exists("%s/scigest/core/CatalogGen.class" % build_path):
        os.environ['CLASSPATH'] = build_path
        os.environ['CLASSPATH'] = "%s:%s" % (build_path, resource_path)
        ext_dirs = "%s/lib/fetched/runtime" % scigest_home()
        print("adding %s to class path" % build_path)
    else:
        raise StandardError("Can't find CatalogGen at %s or %s"
                            % (build_path, jar_path))
    cmd = 'java %s=%s %s=%s scigest.core.CatalogGen' \
          % ("-Dproperties.location",
             user_home() + "/.scigest/application.properties",
             "-Djava.ext.dirs",
             ext_dirs)
    print(cmd)
    proc = subprocess.Popen(cmd.split(" "),
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    result = "".join(proc.stdout.readlines())
    print result
    flist = re.findall(r"Generating catalog file \[([^\]]+)\]", result)
    
    return ['-i', inpath, '-t', '%s' % ",".join(flist)]

# ---------------------------------------------------------------------------
def ez_ingest(argv):
    """ingest - make data available for public downloading

    usage: ezpub ingest [-m <extra-metadata>] [-H <hostname>]

    This is the final publish step. It makes data visible on the ESGF
    portal site. Using the -m/--metadata option, you can attach extra
    metadata to the dataset.
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    p.add_option('-m', '--metadata',
                 action='store', default='', dest='metadata',
                 help='where to find external metadata')
    p.add_option('-H', '--host',
                 action='store', default=REMOTE_HOST, dest='host',
                 help='which esgf portal to talk to')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()

    scigest_home()
    user_home()
    
    validate_hostname(o.host)

    validate_external_metadata(o.metadata)
    
    sp = server_proxy(o.host)
    rc = sp.solr_publish(username(), True)
    if rc != "OK":
        print rc

    if o.metadata != '':
        rc = sp.solr_attach_metadata(username(),
                                     "%s/%s"
                                     % (ESG_DATAROOT,
                                       ezp_cfg.read_app_props("project.name")))
        if rc != "OK":
            print rc

# ---------------------------------------------------------------------------
def ez_list_projects(argv):
    """list_projects - list the available projects

    usage: ezpub list_projects

    List available projects.
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()
    
    # Load the configuration
    esg_cfg = ezp_cfg.loadConfig("%s/ezpub/esg.ini" % scigest_home())

    # Get the list of projects
    projlist = ezp_cfg.get_value_list('project', 'DEFAULT', esg_cfg)

    # Display the list of projects
    list_projects(projlist)

# ---------------------------------------------------------------------------
def ez_list_reqdirs(argv):
    """list_reqdirs - list my request directories on the portal

    usage: ezpub list_reqdirs [-H <hostname>]

    The request directories on the portal owned by the current user
    are listed.
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    p.add_option('-H', '--host',
                 action='store', default=REMOTE_HOST, dest='host',
                 help='which esgf portal to talk to')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()

    validate_hostname(o.host)
    
    sp = server_proxy(o.host)
    rql = sp.list_reqdirs(username())
    if type(rql) == str:
        print rql
        sys.exit(1)
    elif type(rql) == list:
        if 0 < len(rql):
            for rq in rql:
                print("   %s" % rq)
        else:
            print("No request directories")
    else:
        print("Invalid return value of %s" % type(rql))

# ---------------------------------------------------------------------------
def ez_localpub(argv):
    """localpub - publish data to the local node, without staging
    
    usage: ezpub localpub -i <input-dir> -p <project>

    Generate, deploy, and ingest, all in one.
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    p.add_option('-i', '--input',
                 action='store', default='', dest='input',
                 help='path to input file(s)')
    p.add_option('-p', '--project',
                 action='store', default='', dest='project',
                 help='specify the project to publish under')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()

    if o.input == '':
        inpath = interruptible_input("Input path? [%s] > " % default_input)
        inpath = o.path.expandvars(os.path.expanduser(inpath))
        if inpath == '':
            inpath = default_input
    else:
        inpath = o.input

    validate_input_dir(inpath)

    if not os.path.exists(inpath):
        raise StandardError("Input '%s' does not exist" % inpath)
    elif not os.access(inpath, os.R_OK):
        raise StandardError("Input '%s' is not readable" % inpath)

    esg_cfg = ezp_cfg.loadConfig("%s/ezpub/esg.ini" % scigest_home())
    projlist = ezp_cfg.get_value_list('project', 'DEFAULT', esg_cfg)
    if o.project != '':
        project = o.project
        if project not in projlist:
            raise StandardError("Project '%s' is not recognized" % project)
    else:
        project = userSelect('project', projlist)

    section = 'project:' + project
    esg_cfg.set(section, 'project', project)

    require_dot_scigest()
    fpath = write_application_properties(inpath, project, esg_cfg, "")

    catlist = thredds_generate(fpath)

    thredds_xref(catlist)

    thredds_reinit()

    xmllist = [x.strip() for x in contents(catlist, string=False)]
    solr_publish(xmllist)

# ---------------------------------------------------------------------------
def ez_localunpub(argv):
    """localunpub - unpublish data on the local node
    
    usage: ezpub localunpub -x <xmlfile>

    Remove metadata from Solr and the THREDDS catalog.
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    p.add_option('-x', '--xml',
                 action='store', default='', dest='filename',
                 help='xml file(s) to unpublish')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()
    # verify that $SCIGEST_HOME and $HOME are set
    scigest_home()
    user_home()
    
    solr_publish(comma_list(o.filename), ingest=False)

    thredds_xref(comma_list(o.filename), add=False)
    for fname in comma_list(o.filename):
        os.unlink("%s/1/%s" % (THREDDS_ROOT, fname))

    thredds_reinit()

# ---------------------------------------------------------------------------
def ez_publish(argv):
    """publish - make data available for public downloading

    usage: ezpub publish [-i <data directory>] [-p <project>] [-H <host>]
           
    Generate, stage, deploy, and ingest, all in one.

    Options:
     -i/--input <directory>
     indicates the directory where the data can be found.
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    p.add_option('-i', '--input',
                 action='store', default='', dest='input',
                 help='path to input file(s)')
    p.add_option('-m', '--metadata',
                 action='store', default='', dest='metadata',
                 help='where to find external metadata')
    p.add_option('-p', '--project',
                 action='store', default='', dest='project',
                 help='specify the project to publish under')
    p.add_option('-H', '--host',
                 action='store', default=REMOTE_HOST, dest='host',
                 help='which esgf portal to talk to')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()

    validate_input_dir(o.input)
    validate_hostname(o.host)
    validate_external_metadata(o.metadata)
    
    ig_argv = []
    gen_argv = argv
    if '-m' in gen_argv:
        gen_argv.remove(o.metadata)
        gen_argv.remove('-m')
    st_argv = ez_generate(gen_argv)
    if o.host != REMOTE_HOST:
        st_argv.extend(["-H", o.host])
        ig_argv = ["-H", o.host]
    if o.metadata != '':
        st_argv.extend(["-m", o.metadata])
        ig_argv.extend(["-m", o.metadata])
    ez_stage(st_argv)
    ez_ingest(ig_argv)
        
# ---------------------------------------------------------------------------
def ez_stage(argv):
    """stage - copy files to datanode

    usage: ezpub stage -i <data> -t <thredds> [-m <extra-metadata>]
                      [-H <hostname>]
           
    Copy files to data node so once they are published, they can be retrieved.
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    p.add_option('-i', '--input',
                 action='store', default='', dest='input',
                 help='where to find the data files to be copied')
    p.add_option('-m', '--metadata',
                 action='store', default='', dest='metadata',
                 help='where to find external metadata')
    p.add_option('-t', '--thredds',
                 action='store', default='', dest='thredds',
                 help='where to find the thredds file to copy')
    p.add_option('-H', '--host',
                 action='store', default=REMOTE_HOST, dest='host',
                 help='which esgf portal to talk to')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()

    user_home()

    validate_input_dir(o.input)
    validate_thredds_list(o.thredds)
    validate_hostname(o.host)
    validate_external_metadata(o.metadata)
    
    disclaimer()
    load_globus()

    clname, reqdir = make_copy_list(o.input, o.metadata, o.thredds, o.host)

    print(" ")
    print("Running globus-url-copy to move the data...")
    print("You will be prompted for your pin + passcode to authenticate"
          + " for the transfer.")
    print(" ")

    cmd = "globus-url-copy -vb -cd -f %s" % clname
    print(cmd)
    rc = os.system(cmd)
    if rc != 0:
        sys.exit(rc)

    complete_stage(reqdir, o.input, o.thredds, o.host)

# ---------------------------------------------------------------------------
def ez_unpublish(argv):
    """unpublish - make files unavailable for public downloading

    usage: ezpub unpublish {-x <filename> | -r <reqdir>}
           
    Remove specified data from published collection.
    """
    p = OptionParser()
    p.add_option('-d', '--debug',
                 action='store_true', default=False, dest='debug',
                 help='start the debugger')
    p.add_option('-H', '--host',
                 action='store', default=REMOTE_HOST, dest='host',
                 help='which esgf portal to talk to')
    p.add_option('-r', '--reqdir',
                 action='store', default='', dest='reqdir',
                 help='name of request directory')
    p.add_option('-x', '--xml',
                 action='store', default='', dest='xml',
                 help='name of xml file to unpublish')
    (o, a) = p.parse_args(argv)

    if o.debug: pdb.set_trace()

    if o.reqdir != '' and o.xml != '':
        raise StandardError("Only one of -r or -x is allowed.")
    elif o.reqdir == '' and o.xml == '':
        raise StandardError("One of -r or -x is required.")
    
    scigest_home()
    user_home()
    sp = server_proxy(o.host)

    #
    # solr_unpub_files() and solr_unpub_reqdir() remove the indicated
    # records from the solr index.
    #
    # thredds_unpub_files() and thredds_unpub_reqdir() do the following:
    #    call thredds_xref() to remove the references from
    #       /esg/content/thredds/esgcet/catalog.xml
    #    remove the xml files from /esg/content/thredds/esgcet/1
    #
    if o.xml != '':
        sys.stdout.write("Removing files from Solr ... ")
        sys.stdout.flush()
        rc = sp.solr_unpub_files(username(), comma_list(o.xml))
        if rc != "OK":
            print rc
        else:
            print "succeeded!"

        sys.stdout.write("Removing files from THREDDS ... ")
        sys.stdout.flush()
        rc = sp.thredds_unpub_files(username(), comma_list(o.xml))
        if rc != "OK":
            print rc
        else:
            print "succeeded!"
    elif o.reqdir != '':
        sys.stdout.write("Removing files from Solr ... ")
        sys.stdout.flush()
        rc = sp.solr_unpub_reqdir(username(), o.reqdir)
        if rc != "OK":
            print rc
        else:
            print "succeeded!"
        sys.stdout.write("Removing files from THREDDS ... ")
        sys.stdout.flush()
        rc = sp.thredds_unpub_reqdir(username(), o.reqdir)
        if rc != "OK":
            print rc
        else:
            print "succeeded!"

    #
    # reinit the thredds catalog
    #
    sys.stdout.write("Reinitializing the THREDDS server ... ")
    sys.stdout.flush()
    rc = sp.thredds_reinit(username())
    if rc != "OK":
        print rc
    else:
        print "succeeded!"

# ---------------------------------------------------------------------------
def comma_list(string):
    """
    Given a string of the form "item1, item2, item3, ... itemN",
    return a list of the form ["item1", "item2", "item3", ...
    "itemN"]. If there is only one item in the original string, return
    a list containing that single item, eg., [ "item1" ].
    """
    if "," in string:
        rval = re.split("\s*,\s*", string)
    else:
        rval = [ string ]
    return rval

# ---------------------------------------------------------------------------
def complete_stage(reqdir, idir, thredds, thost):
    """
    For each data file, send a "C" request to move the file to its
    final location. For each thredds file, send a "T" request to move
    the file into the thredds catalog.

    reqdir: the (remote) request directory
            ("/tmp/esg/lpad/<user>.<date/time>") to which the files
            have been staged

    idir:   the (local) input directory from which the files were staged

    thredds: 
    """
    # dataroot = get_dataroot("esg_dataroot")
    try:
        sp = server_proxy(thost)
        
        # collect urlPath values from the thredds file(s)
        # updict = parse_urlpaths(thredds)
        
        # for each data file,
#         for fpath in datafile_list(idir):
#             # find the corresponding urlPath value
#             fbase = os.path.basename(fpath)
#             npath = updict[fbase].replace("esg_dataroot", dataroot, 1)
#             rc = sp.copy_data(username(),
#                               "%s/%s/%s" % (LANDING_PAD, reqdir, fbase), npath)
#             if rc != "OK":
#                 print rc
            
        # for each thredds file,
        for tfile in comma_list(thredds):
            rc = sp.thredds_add_file(username(), tfile)
#                                      "%s/%s/%s"
#                                      % (LANDING_PAD,
#                                         reqdir,
#                                         os.path.basename(tfile)))
            if rc != "OK":
                print rc

        # send a request to run CatalogXref
        rc = sp.thredds_xref(username(), "%s/.scigest/catalog.txt" % user_home())
#        rc = sp.thredds_xref(username(), "%s/%s/catalog.txt"
#                             % (LANDING_PAD, reqdir))
        if rc != "OK":
            print rc
        
        # send request to reinitialize the THREDDS server
        rc = sp.thredds_reinit(username())
        if rc != "OK":
            print rc
        
    except Exception, exc:
        print str(exc)
        sys.exit(1)
        
# ---------------------------------------------------------------------------
def datafile_list(dir, xmd_user=""):
    """
    Walk dir and return a list of files matching "*.nc" and "*.cdf".

    If "readme.md", "readme.txt", or "readme.html" exists in the tree,
    include that in the list, preferring the filenames in that order.
    That is, if both readme.txt and readme.html are found, only
    readme.txt will be included.
    """
    xmd_names = supported_metadata_filenames()
    xmd = ""
    rval = []
    for (dpath, dnlist, fnlist) in os.walk(dir):
        for fn in fnlist:
            if fn.endswith(".nc") or fn.endswith(".cdf"):
                rval.append(os.path.abspath("%s/%s" % (dpath, fn)))
            if fn in xmd_names:
                if xmd == "" or xmd_names.index(fn) < xmd_names.index(xmd):
                    xmd = "%s/%s" % (dpath, fn)
    if xmd_user != "":
        rval.append(os.path.abspath(xmd_user))
    elif xmd != "":
        rval.append(os.path.abspath(xmd))
        
    return rval
    
# ---------------------------------------------------------------------------
def disclaimer():
    """
    Display a disclaimer and ask the user to confirm what it says.
    """
    print "*" * 72
    f = open("%s/ezpub/disclaimer.txt" % scigest_home(), "r")
    for line in f:
        sys.stdout.write(line)
    f.close()
    print "*" * 72
    print "Please confirm by typing 'yes' and pressing Enter."
    print "Any other response will terminate the program."
    answer = interruptible_input(" > ")
    if answer.strip().lower() != "yes":
        print("Exiting...")
        sys.exit(1)

# ---------------------------------------------------------------------------
def get_dataroot(drname):
    """
    Find dataroot drname in the configuration and return its value.
    """
    cfg = ezp_cfg.loadConfig("%s/ezpub/esg.ini" % scigest_home())
    roots = cfg.get('DEFAULT', "thredds_dataset_roots")
    for pair in ezp_cfg.splitRecord(roots):
        if pair[0] == drname:
            rval = pair[1]
    return rval

# ---------------------------------------------------------------------------
def interruptible_input(prompt):
    """
    Prompt the user for information and handle keyboard interrupts sensibly.
    """
    try:
        rval = raw_input(prompt)
    except KeyboardInterrupt:
        print "[user abort]"
        sys.exit(1)
    return rval

# ---------------------------------------------------------------------------
def list_projects(plist):
    """
    Display the list of items in the argument.
    """
    print("Available Projects:")
    num = 1
    for proj in plist:
        print("  %d.  %s" % (num, proj))
        num += 1
    print("")
    
# ---------------------------------------------------------------------------
def load_globus():
    """
    Use the module system to set up the environment so we can use
    gridftp to transfer files.
    """
    if None == os.getenv("MODULEPATH"):
        os.environ['MODULEPATH'] = "/sw/home/modulefiles"
    module_load("DefApps")
    module_load("globus/5.0.2")

# ---------------------------------------------------------------------------
def module_load(modname):
    """
    /usr/bin/modulecmd generates a list of strings to set environment
    variables so that the requested module becomes accessible. Here we
    run "/usr/bin/modulecmd bash load globus/5.0.2" to get the
    appropriate settings for globus/5.0.2, and set those variables in
    the environment of the current process.
    """
    cmd = "/usr/bin/modulecmd bash load %s" % modname
    proc = subprocess.Popen(cmd.split(" "),
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    x = "".join(proc.stdout.readlines())
    for piece in x.split(";"):
        # print piece
        if piece.startswith("export"):
            continue
        if "=" in piece:
            (var, val) = piece.split("=")
            os.environ[var] = val.strip()

# ---------------------------------------------------------------------------
def make_copy_list(idir, metadata, thredds, thost):
    """
    Create a file named 'copylist_<pid>.txt' containing a list of
    source and target URLs to control the transfer.
    """
    require_dot_scigest()
    filename = "%s/.scigest/copylist_%d.txt" % (user_home(), os.getpid())

    project  = ezp_cfg.read_app_props("project.name")
    datadir = "%s/%s" % (ESG_DATAROOT, project)
    f = open(filename, "w")
    for fpname in datafile_list(idir, metadata):
        f.write("file:%s sshftp://%s/%s/%s\n"
                % (fpname, thost, datadir, os.path.basename(fpname)))

    rqdir = get_reqdir_name(LANDING_PAD, username())
    tlist = comma_list(thredds)
    for tname in tlist:
        if not os.path.exists(tname):
            raise StandardError("File '%s' not found." % tname)
        f.write("file:%s sshftp://%s/%s/%s\n"
                % (os.path.abspath(tname),
                   thost,
                   rqdir,
                   os.path.basename(tname)))

    home = os.environ['HOME']
    f.write("file:%s/%s sshftp://%s/%s/%s"
            % (user_home(), "/.scigest/catalog.txt",
               thost, rqdir, "catalog.txt"))
            
    f.close()
    return filename, os.path.basename(rqdir)

# ---------------------------------------------------------------------------
def make_empty_catalog():
    """
    Create an empty catalog.xml file.
    """
    require_dot_scigest()
    f = open("%s/.scigest/output/catalog.xml" % user_home(), "w")
    f.write('<?xml version="1.0" encoding="UTF-8"?>\n')
    f.write('<catalog xmlns="http://www.unidata.ucar.edu/namespaces/thredds'
            + '/InvCatalog/v1.0" xmlns:xlink="http://www.w3.org/1999/xlink"'
            + ' name="Earth System Grid catalog" version="1.0.1">\n')
    f.write('  <datasetRoot path="esg_dataroot" location="/esg/data/" />\n')
    f.write('</catalog>\n')
    f.close()
    
# ---------------------------------------------------------------------------
def parse_urlpaths(thredds):
    """
    Scan a collection of thredds files for 'urlPath="(.*)"'. Generate
    and return a dictionary of the matched values, keyed by the basenames. So,

       { 'basename': 'esg_dataroot/foo/bar/baz/basename' }
    """
    rval = {}
    for tfile in comma_list(thredds):
        if not os.path.exists(tfile):
            raise StandardError("File '%s' not found." % tfile)
        q = re.findall('urlPath="([^"]+)"', contents(tfile))
        for path in q:
            rval[os.path.basename(path)] = path
    return rval

# ---------------------------------------------------------------------------
# def pubd_interact(msg, host):
#     """
#     Send msg to pubd, read a response, throw exception if response not "OK".
#     """
#     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#     sock.connect((host, REMOTE_PORT))
#     sock.send(msg + "\n")
#     response = sock.recv(1024)
#     sock.close()
#     if response.strip() != "OK":
#         raise StandardError("Portal: " + response)
    
# ---------------------------------------------------------------------------
def server_proxy(hostname):
    """
    On the first call, create a server proxy, cache it, and return it.
    On subsequent calls, return the cached server proxy object.
    """
    global proxy

    try:
        rval = proxy
    except Exception, e:
        proxy = xmlrpclib.ServerProxy('http://%s:%d' % (hostname, REMOTE_PORT))
        rval = proxy

    return rval

# ---------------------------------------------------------------------------
def supported_metadata_files():
    return ["readme.md", "readme.txt", "readme.html"]

# ---------------------------------------------------------------------------
def userSelect(item_name, vlist):
    """
    Ask the user to select an item from a list.
    """
    if 0 < len(vlist):
        try_again = True
        while try_again:
            # display the list of options
            print(item_name + ':')
            idx = 1
            for val in vlist:
                print("  %d. %s" % (idx, val))
                idx += 1
                
            # ask the user to pick one
            rsp = interruptible_input("\nPlease select a %s > " % item_name)

            # figure out which one the user picked
            if rsp.isdigit():
                pidx = int(rsp)
                if pidx < 1 or len(vlist) < pidx:
                    try_again = True
                else:
                    rval = vlist[pidx-1]
                    try_again = False
            elif rsp in vlist:
                rval = rsp
                try_again = False
            else:
                try_again = True
        return(rval)
    elif 1 == len(vlist):
        return vlist[0]
    else:
        rval = interruptible_input("Please specify a %s > " % item_name)
        return rval

# ---------------------------------------------------------------------------
def validate_external_metadata(fpath):
    """
    If fpath is not empty, it must be one of 'readme.md',
    'readme.txt', or 'readme.html' and it must exist and be readable.
    """
    if fpath == '':
        return

    xmd_list = supported_metadata_files()

    if os.path.basename(fpath) not in xmd_list:
        raise StandardError("External metadata filename must be one of '%s'"
                            % xmd_list)

    if not os.access(fpath, os.R_OK):
        raise StandardError("No such file '%s'" % fpath)

# ---------------------------------------------------------------------------
def validate_hostname(hostname):
    """
    Check that hostname is the name of a valid host.
    """
    if hostname == '':
        raise StandardError("Can't connect to the null host. "
                            + "Please specify a host.")
    socket.gethostbyname(hostname)
    
# ---------------------------------------------------------------------------
def validate_input_dir(dirname):
    """
    If dirname does not exist, is not a directory, or contains no
    ".nc" or ".cdf" files, raise an exception
    """
    if not os.path.isdir(dirname):
        raise StandardError("'%s' is not a directory." % dirname)

    if not os.access(dirname, os.R_OK):
        raise StandardError("Directory '%s' is not readable" % dirname)
    
    dlist = []
    for (dir, dl, fl) in os.walk(dirname):
        for fn in fl:
            if re.search(r"\.nc$", fn) or re.search(r"\.cdf$", fn):
                dlist.append(fn)
    if 0 == len(dlist):
        raise StandardError("No data files found in directory '%s'"% dirname)
    
# ---------------------------------------------------------------------------
def validate_reqdir(dirname):
    """
    Argument should be of the form "<username>.<date/time>". If it is
    not, complain
    """
    if dirname == '':
        return
    x = dirname.split(".", 1)
    p = pwd.getpwnam(x[0])
    ts = time.strptime(x[1], "%Y.%m%d.%H%M%S")
    
# ---------------------------------------------------------------------------
def validate_thredds_list(filename):
    """
    If filename does not exist or does not end with ".xml", raise an exception.

    filename can be a comma-separated list of paths.
    """
    fnlist = comma_list(filename)
    for fn in fnlist:
        if os.path.isdir(fn):
            raise StandardError("'%s' is a directory. -t expects an XML file."
                                % fn)
        elif not fn.endswith(".xml"):
            raise StandardError("'%s' is not an XML file." % fn)
        elif not os.path.exists(fn):
            raise StandardError("'%s' does not exist." % fn)
    
# ---------------------------------------------------------------------------
def write_application_properties(inpath, project, cfg, thost):
    """
    Create an application.properties file for passing informtion into
    scigest.core.CatalogGen when it is run.
    """
    filepath = "%s/.scigest/application.properties" % user_home()
    f = open(filepath, "w")
    f.write("data.src = %s\n" % os.path.abspath(inpath))
    f.write("dir.layout = flat\n")
    f.write("outdir = %s/.scigest/output/1\n" % user_home())
    f.write("\n")
    f.write("project.name = %s\n" % project)
    section = "project:%s" % project

    try:
        f.write("project.model = %s\n" % cfg.get(section, "model"))
    except NoOptionError:
        f.write("project.model = generic\n")
        pass

    try:
        f.write("project.institute = %s\n" % cfg.get(section, "institute"))
    except NoOptionError:
        f.write("project.institute = ORNL\n")

    try:
        f.write("project.tf = %s\n" % cfg.get(section, "time_frequency"))
    except NoOptionError:
        f.write("project.tf = unspecified\n")
        
    f.write("\n")
    f.write("catalog.name = This does not matter\n")
    f.write("catalog.root = %s/.scigest/output\n" % user_home())
    f.write("catalog.link = true\n")
    f.write("catalog.xref = ${catalog.root}/catalog.xml\n")
    f.write("\n")
    f.write("dataset.rootid = %s.%s.\n" % (cfg.get('DEFAULT', 'root_id'), 
                                           project))
    f.write("\n")
    f.write("childdataset.variables = true\n")
    f.write("\n")
    f.write("esg.search.solr.publish.url=http://%s:8983/solr\n" % thost)
    f.write("esg.search.solr.query.url=http://%s:8983/solr\n" % thost)
    f.close()
    return filepath

# ---------------------------------------------------------------------------
if __name__ == '__main__':
    main(sys.argv)
